<?php
/**
 * 
 * 根据话题抓取微博数据
 * @author yangchao
 * @since 2012-03-19
 */

if (!file_exists("../SINASRV_CONFIG")) {
	echo "Read the sinasrv config error!\n";
//	exit();
} else {
   $array_config = parse_ini_file("../SINASRV_CONFIG", true); $_SERVER = array_merge($_SERVER, $array_config); 
}

/*ini_set("display_errors",ture);
error_reporting(E_ALL);*/

define("CRON_NAME","cron_get_rt_top_data_by_level");//cro名称，要求唯一
set_time_limit(0);
$_GET["db_debug"] = true;
$_GET["no_pref_debug"] = true;
//$_GET["api_debug"] = true;

//15分钟一次,执行1000条
define("PCOUNT",10);//每次处理多少mid
define("MAX_PAGE",100); //定义每个进程处理最大的页数
define("TOTAL_PROCESS",1);//添加新进程需要更新这个配置
define("NOW_PROCESS",$argv[1] ? $argv[1] : 0);//当前进程ID=0 可以申请如下php xxx 1
define("CRON_START_TIME",time());



include_once($_SERVER['DOCUMENT_ROOT'] ."/include/user/inc.php");
include_once($_SERVER['DOCUMENT_ROOT'] ."/include/tucao/inc.php");
include_once($_SERVER['DOCUMENT_ROOT'] ."/include/mblog/inc.php");

class mod_cron_engine_main extends mod_cron_engine
{
	function __construct($current_process_id)
	{
		parent::__construct($current_process_id,CRON_NAME);
	}
	
	
	function craw()
	{
		$this->get_data();
	}
	
	
	function  run()
	{
		//开始跑cron之前预先获取数据（start_id等）
		$this->before_run();
		
		//正式获取数据
		$this->craw();
		
		
		$this->after_run();
	}
	
	
	function get_data()
	{
		$res = $this->replace_into_data(0);	
		$db_mblog_rt_mid = new db_mblog_rt_mid();
		
		//获取tag的总数
		$count = $db_mblog_rt_mid->cron_get_mid_count(0);
		
		echo "tag_count:$count<br>\n";
		$total_page = ceil($count/PCOUNT);
		if($total_page > MAX_PAGE && defined("MAX_PAGE")) $total_page = MAX_PAGE;
			
		
		
		
		for($i=1;$i<=$total_page;$i++)
		{
			$this->get_list_by_page($i);
		}
		
		
		$res = $this->replace_into_data(1);	
			
		
	}	
	
	function get_list_by_page($p)
	{
		$db_mblog_rt_mid = new db_mblog_rt_mid();
		
		//分批次取tag名称
		$mids = $db_mblog_rt_mid->cron_get_mid_list_order_by_level($p,PCOUNT);
	
		foreach($mids as $value)
		{
			$mid = $value["mid"];
			
			$start_id = $value["last_get_max_id"];
			$id = $value["id"];
			$last_get_min_id = $value["last_get_min_id"];
			
			if($id % TOTAL_PROCESS != NOW_PROCESS)//用总进程做控制 example: 3个进程  php xxx 1 只会执行除3余1的id
			{
				continue;
			}
			
			if($mid)//处理抓取分页逻辑
			{
				$this->get_list_by_topic_by_new($id,$mid); //抓取最新的转发
				
			}
			
		
			if($last_get_min_id != -1) //-1的时候表示已经全抓抓完
			{
				$this->get_list_by_topic_by_old($id,$mid,$last_get_min_id);//专区老的转发如果还没有抓取完
			}
			
			
			
			
		}
		
		
	}
	
	
	//根据话题抓取数据并入库
	function get_list_by_topic_by_new($id,$mid,$start_id="",$p="")
	{
		if($p < 1) $p=1;
		
		$db_base = new db_base();
		
		$max_time = "";
		//执行时间超过30分钟，退出
		if(time() - CRON_START_TIME > 300)
		{
			exit;
		}
		
		if(!$start_id && $p ==1)
		{
			//开始抓取的id
			$db_mblog_rt_count= new db_mblog_rt_count();
			$mid_info = $db_mblog_rt_count->cron_get_max_status_id_by_mid($mid);
			if($mid_info)$start_id = $mid_info["status_id"];
		}
	
		
		$sql =" insert ignore into  mblog_rt_count (mid,status_id,status) values ";
		$sql_array = array();
		
		$mblog_from_api = new mblog_from_api();
		$mblog_list = $mblog_from_api->api_get_rt_ids_by_mid($mid,$start_id,"",$p,200);
		$mblog_list = $mblog_list["data"]["result"];
		
		foreach($mblog_list as $value)
		{
			$sql_array[] = " ($mid,$value,1) ";
		}
		
		$db_mblog_rt_mid= new db_mblog_rt_mid();
		if($sql_array)
		{
			$sql .= join(",",$sql_array);
			$res = $db_base->query($sql);
		}
		else
		{
			if($p == 1)//如果第一页就没有抓取到数据，更新level，抓取优先级降低1
			{
				$db_mblog_rt_mid->cron_update_by_id($mid,-1);
			}
			
		}
		
		//如果个数大于150就算没有结束，继续抓下一页
		if(count($mblog_list) > 150)
		{
			$p = $p+1;
			$this->get_list_by_topic_by_new($id,$mid,$start_id,$p);
		}
		
			
	}
	
	
	
	
	

	//抓取上次抓取可能没有抓取完毕的转发数据
	function get_list_by_topic_by_old($id,$mid,$end_id="",$p="")
	{
		if($p < 1) $p=1;
		
		$db_base = new db_base();
		
		$max_time = "";
		//执行时间超过30分钟，退出
		if(time() - CRON_START_TIME > 300)
		{
			exit;
		}
		
		$sql =" insert ignore into  mblog_rt_count (mid,status_id,status) values ";
		$sql_array = array();
		
		$mblog_from_api = new mblog_from_api();
		$mblog_list = $mblog_from_api->api_get_rt_ids_by_mid($mid,"",$end_id,$p,200);
		$mblog_list = $mblog_list["data"]["result"];
		
		foreach($mblog_list as $value)
		{
			$sql_array[] = " ($mid,$value,1) ";
		}
		
		$db_mblog_rt_mid= new db_mblog_rt_mid();
		if($sql_array)
		{
			$sql .= join(",",$sql_array);
			$res = $db_base->query($sql);//入库后更新最后一次更新的抓取mid
			$db_mblog_rt_mid->cron_update_by_id($mid,"",$mblog_list[count($mblog_list)-1]);
		}
		else
		{
			if($p == 1)//如果第一页就没有抓取到数据，更新为以后不需要抓取
			{
				$db_mblog_rt_mid->cron_update_by_id($mid,"",-1);
			}
			
		}
		
		//如果个数大于150就算没有结束，继续抓下一页
		if(count($mblog_list) > 150)
		{
			$p = $p+1;
			$this->get_list_by_topic_by_new($id,$mid,$end_id,$p);
		}
		
			
	}
}

$m = new mod_cron_engine_main(NOW_PROCESS);
$m->run();
